import hashlib
from datetime import datetime
import time
from bs4 import BeautifulSoup
import requests
import httpx
from entitys.BaiduEntity import BaiduData
from entitys.KeyWordsEntity import KeyWords


def linkMd5(link):
    return hashlib.md5(link.encode()).hexdigest()


def get_html(url, keywords):
    # 发送请求获取网页内容
    client = httpx.Client(http2=True)
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36',
        'X-Requested-With': 'XMLHttpRequest',
        'Cookie': 'BIDUPSID=457C3455302D430C9674CAC89AF78CFA; PSTM=1657781295; BAIDUID=CE87CE2967A79BC2C0617B8BA1AC45B3:SL=0:NR=10:FG=1; BD_UPN=12314753; MCITY=-176%3A; H_WISE_SIDS_BFESS=234020_216839_213354_214792_110085_244727_259031_261710_236312_256419_265881_266354_265615_267074_268405_268593_266186_259642_269232_269783_268236_269905_267066_256739_270460_270519_270603_270547_271019_271173_256958_267659_271319_270102_271879_269857_271813_271935_234296_234208_269296_272082_272226_270055_272279_267596_272336_253022_272653_272836_260335_272986_273061_267558_273160_273164_273138_273215_273263_273238_273301_273400_273392_271158_273455_273473_273519_271562_271146_273703_273318_264170_272262_263619_274082_273931_273965_274142_269610_273918_273044_273598_272695_272859_256223_272562_274426_274441_188333_274649_274765_274759_274822_274839_274844_274856_274847_270158_274870_273981_275070_272801_274452_267807_267547_275145_272333_275167_275197_275147_272318_8000051_8000108_8000124_8000142_8000160_8000162_8000165_8000170_8000183_8000186_8000204; sug=3; sugstore=0; ORIGIN=0; bdime=0; ispeed_lsm=2; H_WISE_SIDS=39635_39668_39672_39664_39676_39679_39712_39751_39766_39791_39704_39688; BDUSS=lJMeHZRYmRpSXE0UGp6dk5USW5RaXd2VXRYcFNrVFFzUk5SeEh1aEtYaVd0WUpsRVFBQUFBJCQAAAAAAAAAAAEAAACfa9v~c3hsYW94aWFuZwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAJYoW2WWKFtlf; BDUSS_BFESS=lJMeHZRYmRpSXE0UGp6dk5USW5RaXd2VXRYcFNrVFFzUk5SeEh1aEtYaVd0WUpsRVFBQUFBJCQAAAAAAAAAAAEAAACfa9v~c3hsYW94aWFuZwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAJYoW2WWKFtlf; BDORZ=B490B5EBF6F3CD402E515D22BCDA1598; BDSFRCVID=5uIOJeC62mpfkbJqVZgnUAN5gSNgBovTH6aozrIKbiVyO4D2LviLEG0PFM8g0KFhOdh3ogKKyeOTHu8F_2uxOjjg8UtVJeC6EG0Ptf8g0x5; H_BDCLCKID_SF=Jn-O_C82JKL3HJOmMJjE5bcHMhoK--Q0KKJyLR-8Kb7VbPjoMUnkbfJBDGOUQbjkK56e2JRkQIbvennJhf6AXx47yajK2boGQG58-UbuQtoFDqrP0UjpQT8ryJ_OK5OiJgDqslnNab3vOPTNXpO15jkzBN5thURB2DkO-4bCWJ5TMl5jDh3Mb6ksDMDtqj0OtbkO_Ktyf-3bfTrx5tQHMDCShGRHafo9WDTm_D_5H4JdjKQl0nJcKh33yP6zK-o30K6Z-pPKKR70sPjoht6obj8VMnrrBJcB3mkjbpQGfn02OP5EQUrcW44syPR8KxRnWTIOKfA-b4ncjRcTehoM3xI8LNj405OTbIFO0KJyJCFhhKPGD6uMD5PfbeTXatb8-bCX3buQQh4V8pcNLTDK5fAq5UAtblJM-n6n_pvFytQaDt5kjlO1j4_eKpOg0fbhyTQJ-l4aytOzSh5jDh3PXjksD-RlanJtKeoy0hvcMR5cShnGeMjrDRLbXU6BK5vPbNcZ0l8K3l02V-bIe-t225QhDG-DJTkjJR-s_C8atTrWeIbcbtI_-P4DePO73xRZ5mAqoqOHKfT4V4jKyU5J350UyR6MBMne-C7naIQqah0WVhOTbMjsqj53XqbNWJO43bRTW-Py5KJvfJopKUTChP-Uypr-Wh37bRblMKoaMp78jR093JO4y4Ldj4oxJpOJ5JbMonLafDKahK-ljjK-eP0OqxJO243fHD7XVK5aWl7keq8CD4c2b4PVylQpXtbXQm6w_R3RBJ3IDPQ2y5jHhnIXytRaal58BbnRoqRK2lRpsIJMQxFWbT8UQx79KPkeaKvia-THBMb1fDnDBT5h2M4qMxtOLR3pWDTm_q5TtUJMeCnTDbAhe4tX-NFJJ5KtJx5; H_PS_PSSID=39635_39668_39672_39664_39676_39712_39751_39766_39791_39704_39688_39679_39817; ab_sr=1.0.1_NTFjOTQzZjRmMjNmMzRkYTgyNjllOGRjNGU4ZDY5ZDQ3ODk5MjhkNjgzMzdiNmQ1ZGU2ZDI0OTgyZmRmYjRhYzZjMzZmYWU1Y2NjNTYxOWJmOTBjYjY3YjJjZmQzODJiY2JiZGUzYTk4NzJlMDRhNjc2YjAxZmFlYWNlNDgxNDU0ZDExNzU5MWEzZTNmZWZlZTUzMjNmZmQwNDk2Y2ViMA==; BA_HECTOR=ah8405018lak2l2lah2ha4241iluc3a1r; delPer=0; BD_CK_SAM=1; PSINO=2; ZFY=ZdvYHXrQ8EZDujNL9Sq:Bs:BYXff6ZUuWUsc:BA5Gx97Qc:C; BAIDUID_BFESS=CE87CE2967A79BC2C0617B8BA1AC45B3:SL=0:NR=10:FG=1; BDSFRCVID_BFESS=5uIOJeC62mpfkbJqVZgnUAN5gSNgBovTH6aozrIKbiVyO4D2LviLEG0PFM8g0KFhOdh3ogKKyeOTHu8F_2uxOjjg8UtVJeC6EG0Ptf8g0x5; H_BDCLCKID_SF_BFESS=Jn-O_C82JKL3HJOmMJjE5bcHMhoK--Q0KKJyLR-8Kb7VbPjoMUnkbfJBDGOUQbjkK56e2JRkQIbvennJhf6AXx47yajK2boGQG58-UbuQtoFDqrP0UjpQT8ryJ_OK5OiJgDqslnNab3vOPTNXpO15jkzBN5thURB2DkO-4bCWJ5TMl5jDh3Mb6ksDMDtqj0OtbkO_Ktyf-3bfTrx5tQHMDCShGRHafo9WDTm_D_5H4JdjKQl0nJcKh33yP6zK-o30K6Z-pPKKR70sPjoht6obj8VMnrrBJcB3mkjbpQGfn02OP5EQUrcW44syPR8KxRnWTIOKfA-b4ncjRcTehoM3xI8LNj405OTbIFO0KJyJCFhhKPGD6uMD5PfbeTXatb8-bCX3buQQh4V8pcNLTDK5fAq5UAtblJM-n6n_pvFytQaDt5kjlO1j4_eKpOg0fbhyTQJ-l4aytOzSh5jDh3PXjksD-RlanJtKeoy0hvcMR5cShnGeMjrDRLbXU6BK5vPbNcZ0l8K3l02V-bIe-t225QhDG-DJTkjJR-s_C8atTrWeIbcbtI_-P4DePO73xRZ5mAqoqOHKfT4V4jKyU5J350UyR6MBMne-C7naIQqah0WVhOTbMjsqj53XqbNWJO43bRTW-Py5KJvfJopKUTChP-Uypr-Wh37bRblMKoaMp78jR093JO4y4Ldj4oxJpOJ5JbMonLafDKahK-ljjK-eP0OqxJO243fHD7XVK5aWl7keq8CD4c2b4PVylQpXtbXQm6w_R3RBJ3IDPQ2y5jHhnIXytRaal58BbnRoqRK2lRpsIJMQxFWbT8UQx79KPkeaKvia-THBMb1fDnDBT5h2M4qMxtOLR3pWDTm_q5TtUJMeCnTDbAhe4tX-NFJJ5KtJx5; H_PS_645EC=2e7cTUbugOqhTVjLNzr5%2FDiuawzqN3FtHO%2Bk%2BIlcJWq46kZzYLKgFusysCA; COOKIE_SESSION=171_1_7_7_8_24_1_0_6_7_1_7_591_0_0_0_1700736994_1700706226_1700737162%7C9%23175549_11_1700706222%7C9; WWW_ST=1700737266076'
    }

    response = client.get(url, headers=headers)
    html_content = response.content

    # 使用BeautifulSoup解析网页内容
    soup = BeautifulSoup(html_content, 'html.parser')
    # 获取新闻列表
    sdata = soup.find_all('div', class_='result-op c-container xpath-log new-pmd')
    for item in sdata:
        Title = item.find('h3', class_='news-title_1YtI1').text
        Source = item.find('span', class_='c-color-gray').text
        Summary = item.find('span', class_='c-font-normal c-color-text').text
        SendTime = item.find('span', class_='c-color-gray2 c-font-normal c-gap-right-xsmall').text if item.find(
            'span',
            class_='c-color-gray2 c-font-normal c-gap-right-xsmall') is not None else None
        link = item.find('a', class_='source-link_Ft1ov').get('href')
        imgUrl = item.find('img', class_='source-img_33bs5').get('src') if item.find('img',
                                                                                     class_='source-img_33bs5') is not None else None
        # 将当前时间转为时间戳
        addTime = time.time()
        baidu = BaiduData(Title=Title, Source=Source, Summary=Summary, SendTime=SendTime, link=link, linkMd5=linkMd5(link), imgUrl=imgUrl, keywords=keywords, addTime=addTime)
        lists = baidu.getListByLinkMd5()
        # 如果lists为空，则保存数据
        if lists:
            continue
        else:
            baidu.save()


def getPages(url):
    p = 0
    # 发送请求获取网页内容
    client = httpx.Client(http2=True)
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36',
        'X-Requested-With': 'XMLHttpRequest',
        'Cookie': 'BIDUPSID=457C3455302D430C9674CAC89AF78CFA; PSTM=1657781295; BAIDUID=CE87CE2967A79BC2C0617B8BA1AC45B3:SL=0:NR=10:FG=1; BD_UPN=12314753; MCITY=-176%3A; H_WISE_SIDS_BFESS=234020_216839_213354_214792_110085_244727_259031_261710_236312_256419_265881_266354_265615_267074_268405_268593_266186_259642_269232_269783_268236_269905_267066_256739_270460_270519_270603_270547_271019_271173_256958_267659_271319_270102_271879_269857_271813_271935_234296_234208_269296_272082_272226_270055_272279_267596_272336_253022_272653_272836_260335_272986_273061_267558_273160_273164_273138_273215_273263_273238_273301_273400_273392_271158_273455_273473_273519_271562_271146_273703_273318_264170_272262_263619_274082_273931_273965_274142_269610_273918_273044_273598_272695_272859_256223_272562_274426_274441_188333_274649_274765_274759_274822_274839_274844_274856_274847_270158_274870_273981_275070_272801_274452_267807_267547_275145_272333_275167_275197_275147_272318_8000051_8000108_8000124_8000142_8000160_8000162_8000165_8000170_8000183_8000186_8000204; sug=3; sugstore=0; ORIGIN=0; bdime=0; ispeed_lsm=2; H_WISE_SIDS=39635_39668_39672_39664_39676_39679_39712_39751_39766_39791_39704_39688; BDUSS=lJMeHZRYmRpSXE0UGp6dk5USW5RaXd2VXRYcFNrVFFzUk5SeEh1aEtYaVd0WUpsRVFBQUFBJCQAAAAAAAAAAAEAAACfa9v~c3hsYW94aWFuZwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAJYoW2WWKFtlf; BDUSS_BFESS=lJMeHZRYmRpSXE0UGp6dk5USW5RaXd2VXRYcFNrVFFzUk5SeEh1aEtYaVd0WUpsRVFBQUFBJCQAAAAAAAAAAAEAAACfa9v~c3hsYW94aWFuZwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAJYoW2WWKFtlf; BDORZ=B490B5EBF6F3CD402E515D22BCDA1598; BDSFRCVID=5uIOJeC62mpfkbJqVZgnUAN5gSNgBovTH6aozrIKbiVyO4D2LviLEG0PFM8g0KFhOdh3ogKKyeOTHu8F_2uxOjjg8UtVJeC6EG0Ptf8g0x5; H_BDCLCKID_SF=Jn-O_C82JKL3HJOmMJjE5bcHMhoK--Q0KKJyLR-8Kb7VbPjoMUnkbfJBDGOUQbjkK56e2JRkQIbvennJhf6AXx47yajK2boGQG58-UbuQtoFDqrP0UjpQT8ryJ_OK5OiJgDqslnNab3vOPTNXpO15jkzBN5thURB2DkO-4bCWJ5TMl5jDh3Mb6ksDMDtqj0OtbkO_Ktyf-3bfTrx5tQHMDCShGRHafo9WDTm_D_5H4JdjKQl0nJcKh33yP6zK-o30K6Z-pPKKR70sPjoht6obj8VMnrrBJcB3mkjbpQGfn02OP5EQUrcW44syPR8KxRnWTIOKfA-b4ncjRcTehoM3xI8LNj405OTbIFO0KJyJCFhhKPGD6uMD5PfbeTXatb8-bCX3buQQh4V8pcNLTDK5fAq5UAtblJM-n6n_pvFytQaDt5kjlO1j4_eKpOg0fbhyTQJ-l4aytOzSh5jDh3PXjksD-RlanJtKeoy0hvcMR5cShnGeMjrDRLbXU6BK5vPbNcZ0l8K3l02V-bIe-t225QhDG-DJTkjJR-s_C8atTrWeIbcbtI_-P4DePO73xRZ5mAqoqOHKfT4V4jKyU5J350UyR6MBMne-C7naIQqah0WVhOTbMjsqj53XqbNWJO43bRTW-Py5KJvfJopKUTChP-Uypr-Wh37bRblMKoaMp78jR093JO4y4Ldj4oxJpOJ5JbMonLafDKahK-ljjK-eP0OqxJO243fHD7XVK5aWl7keq8CD4c2b4PVylQpXtbXQm6w_R3RBJ3IDPQ2y5jHhnIXytRaal58BbnRoqRK2lRpsIJMQxFWbT8UQx79KPkeaKvia-THBMb1fDnDBT5h2M4qMxtOLR3pWDTm_q5TtUJMeCnTDbAhe4tX-NFJJ5KtJx5; H_PS_PSSID=39635_39668_39672_39664_39676_39712_39751_39766_39791_39704_39688_39679_39817; ab_sr=1.0.1_NTFjOTQzZjRmMjNmMzRkYTgyNjllOGRjNGU4ZDY5ZDQ3ODk5MjhkNjgzMzdiNmQ1ZGU2ZDI0OTgyZmRmYjRhYzZjMzZmYWU1Y2NjNTYxOWJmOTBjYjY3YjJjZmQzODJiY2JiZGUzYTk4NzJlMDRhNjc2YjAxZmFlYWNlNDgxNDU0ZDExNzU5MWEzZTNmZWZlZTUzMjNmZmQwNDk2Y2ViMA==; BA_HECTOR=ah8405018lak2l2lah2ha4241iluc3a1r; delPer=0; BD_CK_SAM=1; PSINO=2; ZFY=ZdvYHXrQ8EZDujNL9Sq:Bs:BYXff6ZUuWUsc:BA5Gx97Qc:C; BAIDUID_BFESS=CE87CE2967A79BC2C0617B8BA1AC45B3:SL=0:NR=10:FG=1; BDSFRCVID_BFESS=5uIOJeC62mpfkbJqVZgnUAN5gSNgBovTH6aozrIKbiVyO4D2LviLEG0PFM8g0KFhOdh3ogKKyeOTHu8F_2uxOjjg8UtVJeC6EG0Ptf8g0x5; H_BDCLCKID_SF_BFESS=Jn-O_C82JKL3HJOmMJjE5bcHMhoK--Q0KKJyLR-8Kb7VbPjoMUnkbfJBDGOUQbjkK56e2JRkQIbvennJhf6AXx47yajK2boGQG58-UbuQtoFDqrP0UjpQT8ryJ_OK5OiJgDqslnNab3vOPTNXpO15jkzBN5thURB2DkO-4bCWJ5TMl5jDh3Mb6ksDMDtqj0OtbkO_Ktyf-3bfTrx5tQHMDCShGRHafo9WDTm_D_5H4JdjKQl0nJcKh33yP6zK-o30K6Z-pPKKR70sPjoht6obj8VMnrrBJcB3mkjbpQGfn02OP5EQUrcW44syPR8KxRnWTIOKfA-b4ncjRcTehoM3xI8LNj405OTbIFO0KJyJCFhhKPGD6uMD5PfbeTXatb8-bCX3buQQh4V8pcNLTDK5fAq5UAtblJM-n6n_pvFytQaDt5kjlO1j4_eKpOg0fbhyTQJ-l4aytOzSh5jDh3PXjksD-RlanJtKeoy0hvcMR5cShnGeMjrDRLbXU6BK5vPbNcZ0l8K3l02V-bIe-t225QhDG-DJTkjJR-s_C8atTrWeIbcbtI_-P4DePO73xRZ5mAqoqOHKfT4V4jKyU5J350UyR6MBMne-C7naIQqah0WVhOTbMjsqj53XqbNWJO43bRTW-Py5KJvfJopKUTChP-Uypr-Wh37bRblMKoaMp78jR093JO4y4Ldj4oxJpOJ5JbMonLafDKahK-ljjK-eP0OqxJO243fHD7XVK5aWl7keq8CD4c2b4PVylQpXtbXQm6w_R3RBJ3IDPQ2y5jHhnIXytRaal58BbnRoqRK2lRpsIJMQxFWbT8UQx79KPkeaKvia-THBMb1fDnDBT5h2M4qMxtOLR3pWDTm_q5TtUJMeCnTDbAhe4tX-NFJJ5KtJx5; H_PS_645EC=2e7cTUbugOqhTVjLNzr5%2FDiuawzqN3FtHO%2Bk%2BIlcJWq46kZzYLKgFusysCA; COOKIE_SESSION=171_1_7_7_8_24_1_0_6_7_1_7_591_0_0_0_1700736994_1700706226_1700737162%7C9%23175549_11_1700706222%7C9; WWW_ST=1700737266076'
    }

    response = client.get(url, headers=headers)
    html_content = response.content

    # 使用BeautifulSoup解析网页内容
    soup = BeautifulSoup(html_content, 'html.parser')
    pageInner = soup.find('div', class_='page-inner') if soup.find('div', class_='page-inner') is not None else None
    if pageInner.find_all('a') is not None:
        pages = pageInner.find_all('a') if pageInner.find_all('a') is not None else None
        for page in pages:
            p = page.find('span', 'pc').text if page.find('span', 'pc') is not None else p

    return p


def getBaidu(keyword):
    print(keyword)
    p = getPages(
        'https://www.baidu.com/s?ie=utf-8&medium=0&rtt=4&bsst=1&rsv_dl=news_t_sk&cl=2&wd=' + keyword + '&tn=news&rsv_bp=1&oq=&rsv_btype=t&f=8')
    pn = 0
    # 循环请求
    for i in range(int(p)):
        try:
            url = 'https://www.baidu.com/s?tn=news&rtt=4&bsst=1&cl=2&wd=' + keyword + '&medium=0&tngroupname=organic_news&newVideo=12&goods_entry_switch=1&rsv_dl=news_b_pn&pn=' + str(
                pn)
            print('正在爬取的页面：%s' % url)
            get_html(url, keyword)
            time.sleep(1)
            pn += 10
        except requests.exceptions.RequestException as e:
            print(e)


def getKeywords():
    keywordList = KeyWords.getList()
    for list in keywordList:
        keyword = list[1]
        create_time = list[2]
        getBaidu(keyword)
        # 更新该数据的update_time
        update_time = datetime.now()
        keywords = KeyWords(keyword, create_time, update_time)
        KeyWords.updateTime(keywords)


if __name__ == '__main__':
    # getBaidu("忻州")
    getKeywords()
